package com.minispider;

import java.sql.SQLException;
import java.util.ArrayList;

import com.minispider.entity.ParamEntity;

import com.minispider.util.Db;
import com.minispider.util.HtmlDocument;
import com.minispider.util.Logger;
import com.minispider.util.Utility;
import com.mysql.jdbc.Connection;
import com.mysql.jdbc.ResultSet;
import com.mysql.jdbc.Statement;

public class MiniSpider {

	public static void main(String[] args) throws Throwable {
		
		ArrayList<ParamEntity> pes = getInitParamsList(Utility.mysql_url, Utility.mysql_user, Utility.mysql_password);

/*		
if (true) {
	
	for (int i = 0; i < pes.size(); i++) {
		pes.get(i).print();
	}
	
	return;		
}
*/
		
		for (int i = 0; i < pes.size(); i++) {
			MiniSpider.lookUp(pes.get(i));
		}
	}

	static void lookUp(ParamEntity pe) throws Throwable {
		
		pe.db = Db.getInstance(Utility.mysql_url, Utility.mysql_user, Utility.mysql_password);

		HtmlDocument hd = new HtmlDocument(pe.url, pe.charsetEncoding);
		
		do {
			//
			Logger.info(pe.url);
		
			if (hd.load(pe.url, pe.referer) == null) {
				break;
			}
			//System.out.println(hd.getWellFormedHtmlDoc());
			
			pe.getNextPageHref(hd);
			
			//
		    for (int i = 0; i < pe.xPathExps.get(1).size(); i++) {
				pe.data.add(hd.getHtmlDocNodesByXPathExp(pe.xPathExps.get(1).get(i)));
		    }
			
			pe.process();
		    
			//Thread.sleep(1000); //???
			
		} while (pe.hasMorePage());	
		
		pe.db.release();

	}
	
	static ArrayList<ParamEntity> getInitParamsList(String url, String user, String pwd) throws Throwable {

		ArrayList<ParamEntity> pes = new ArrayList<ParamEntity>();
		
		Db db = Db.getInstance(url, user, pwd);
		
	    Connection con = db.getConnection();

	    try {	  
	    	//
	    	Statement wsiStmt = (Statement) con.createStatement();
	    	ResultSet wsiRs = (ResultSet) wsiStmt.executeQuery("select * from website_info;");
			
	    	while (wsiRs.next()) {				 
				 //
				 Statement ciStmt = (Statement) con.createStatement();
				 ResultSet ciRs = (ResultSet) ciStmt.executeQuery(
						 "select a.*, b.name as classname from channel_info a, class_info b where a.wid="
						 	+ wsiRs.getInt("wid") 
								+ " and a.clsid=b.clsid;");
				 //
				 while (ciRs.next()) {
					 
					ParamEntity pe = (ParamEntity) Class.forName(ciRs.getString("classname")).newInstance();
					
					pe.ids.add(String.valueOf(wsiRs.getInt("wid")));
					pe.ids.add(String.valueOf(ciRs.getInt("cid")));
					pe.name.add(wsiRs.getString("name"));
					pe.name.add(ciRs.getString("name"));
					
					pe.url = ciRs.getString("url");
					pe.referer = "";
					pe.charsetEncoding = ciRs.getString("charset");
					pe.timeStamp = ciRs.getDate("timestamp");
					pe.maxPage = ciRs.getInt("maxpage");
					
					//
					 ArrayList<ArrayList<String>> xPathExps = new ArrayList<ArrayList<String>>();
						
					 Statement xpiStmt = (Statement) con.createStatement();
					 
					 for (int i = 0; i < 2; i++) {
						 ResultSet xpiRs = (ResultSet) xpiStmt.executeQuery(
								"select a.*, b.xpath from channel_info a, xpath_info b where a.wid=" 
								 		+ ciRs.getInt("wid")
						 				+ " and a.cid=" + ciRs.getInt("cid") 
						 				+ " and a.wid=b.wid and a.xid=b.xid and xtype=" + i
						 				+ " order by idx;");

						 ArrayList<String> al = new ArrayList<String>();
						
						 while (xpiRs.next()) {
							 al.add(xpiRs.getString("xpath"));
						 }
						 xpiRs.close();
						 
						 if (al.size() == 0) {
							 xPathExps.add(null);
						 }
						 else {
							 xPathExps.add(al);
						 }
					 }
					 xpiStmt.close();	
					 
					 
					pe.xPathExps = xPathExps;
					
					//
					pes.add(pe);
						 
				 }
				 ciRs.close();
				 ciStmt.close();
			 } 		  
			 wsiRs.close();
			 wsiStmt.close();
	    }
	    catch (SQLException ex)
	    {
			ex.printStackTrace();
	    }
	    
	    db.release();

	    return pes;
	}	
	
}
